from scrapy.utils.misc import load_object
from scrapy.utils.serialize import ScrapyJSONEncoder
from twisted.internet.threads import deferToThread
from . import connection, defaults
from . import spider_report
import json
import uuid
import time

default_serialize = ScrapyJSONEncoder().encode

class WeiboPipeline(object):
    def __init__(self, server,
                 key=defaults.PIPELINE_KEY,
                 serialize_func=default_serialize):
        self.server = server
        self.key = key
        self.serialize = serialize_func
        self.client_id = uuid.uuid1()

    @classmethod
    def from_settings(cls, settings):
        params = {
            'server': connection.from_settings(settings),
        }

        if settings.get('REDIS_ITEMS_KEY'):
            params['key'] = settings['REDIS_ITEMS_KEY']
        if settings.get('REDIS_ITEMS_SERIALIZER'):
            params['serialize_func'] = load_object(
                settings['REDIS_ITEMS_SERIALIZER']
            )

        return cls(**params)

    @classmethod
    def from_crawler(cls, crawler):
        return cls.from_settings(crawler.settings)

    def process_item(self, item, spider):
        return deferToThread(self._process_item, item, spider)

    def _process_item(self, item, spider):
        key = self.item_key(item, spider)
        #data = self.serialize(item)
        data = item['html']
        self.server.rpush(key, data)

        cnt = 1
        tm = str(time.strftime('%Y/%m/%d %H:%M:%S',time.localtime(time.time())))
        spider_report.report(self.client_id, "weibo_comment", tm, cnt)

        return item

    def item_key(self, item, spider):
        return self.key % {'spider': spider.name}
